package com.wontube.smr.bo.sites.crunchyroll;

import java.io.IOException;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.apache.ibatis.session.SqlSession;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wontube.client.extension.utils.DateUtils;
import com.wontube.client.extension.utils.NumberUtils;
import com.wontube.client.extension.utils.StringUtils;
import com.wontube.core.dao.mybatis.QueryExample;
import com.wontube.smr.bo.constants.SiteConstants;
import com.wontube.smr.bo.constants.StandarAreas;
import com.wontube.smr.client.model.po.RecentShow;
import com.wontube.smr.client.model.po.TVEpisode;
import com.wontube.smr.client.model.po.TVSeason;
import com.wontube.smr.client.model.po.TVShow;
import com.wontube.smr.client.model.po.TVShowGenre;
import com.wontube.smr.client.model.po.TVShowToday;
import com.wontube.smr.dao.mapper.RecentShowMapper;
import com.wontube.smr.dao.mapper.TVEpisodeMapper;
import com.wontube.smr.dao.mapper.TVSeasonMapper;
import com.wontube.smr.dao.mapper.TVShowGenreMapper;
import com.wontube.smr.dao.mapper.TVShowMapper;
import com.wontube.smr.dao.mapper.TVShowTodayMapper;
import com.wontube.smr.pub.DefaultDBHelper;

/**
 * Crunchyroll 抓取
 * 
 * @author hewc@wondershare.cn
 * 
 */
public final class CrunchyrollObtainBO {

	private static final String CRUNCHYROLL_ANIME_URL_FORMAT = "http://www.crunchyroll.com/videos/anime/popular/ajax_page?pg={}";
	private static final String CRUNCHYROLL_ANIME_GENRE_URL = "http://www.crunchyroll.com/videos/anime/genres/ajax_page?pg={}&tagged={}";
	private static final String CRUNCHYROLL_ANIME_SIMULCASTS_URL = "http://www.crunchyroll.com/videos/anime/simulcasts/ajax_page?pg={}";
	private static final String CRUNCHYROLL_ANIME_UPDATE_URL = "http://www.crunchyroll.com/videos/anime/updated/ajax_page?pg={}";
	
	private static final String CRUNCHYROLL_DRAMA_URL_FORMAT = "http://www.crunchyroll.com/videos/drama/popular/ajax_page?pg={}";
	private static final String CRUNCHYROLL_DRAMA_GENRE_URL = "http://www.crunchyroll.com/videos/drama/genres/ajax_page?pg={}&tagged={}";
	
	private static final String COMING_SOON_IMAGE = "http://static.ak.crunchyroll.com/i/coming_soon_beta_wide.jpg";
	
	private static final Logger logger = LoggerFactory.getLogger(CrunchyrollObtainBO.class);
	
	private static final String DOMAIN = "http://www.crunchyroll.com";
	
	private final static int CONNECT_READ_TIMEOUT = 60 * 1000;
	private final static String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1";
	private final static String REFERRER = "http://www.crunchyroll.com/";
	
	private static Map<String, String> dramaCountryCates = new HashMap<String, String>();	
	static {
		dramaCountryCates.put("cdrama", TVShow.CATEGORY_CHINESE_TV);
		dramaCountryCates.put("jdrama", TVShow.CATEGORY_JAPANESE_TV);
		dramaCountryCates.put("kdrama", TVShow.CATEGORY_KOREAN_TV);
	}
	
	/**
	 * grab all animes
	 * @return
	 */
	public static int saveAllAnimes(){
		return saveAnimesOrDramas(TVShow.CATEGORY_ANIME);
	}
	
	/**
	 * 更新genre信息-> tvshow, tvshow_genre
	 * @return
	 */
	public static int updateAnimeGenres(){
		int count = 0;
		for (String genre : CrunchyrollAnimeGenres.getInitAnimeGenres()) {
			String trueGenre = CrunchyrollAnimeGenres.remendGenre(genre);		
			
			int page = 0;			
			while(true){
				String url = StringUtils.format(CRUNCHYROLL_ANIME_GENRE_URL, page++, genre);
				int saved = grabAndUpdateTVShowGenres(url, trueGenre);
				if(saved == 0){
					break;
				}else{
					count += saved;
				}
			}
		}
		
		return count;
	}
	
	/**
	 * update anime simulcast to tvshow_today
	 * @return
	 */
	public static int saveAnimeSimulcasts(){
		int count = 0;
		int page = 0;		
		boolean run = true;
		while(run){
			String url = StringUtils.format(CRUNCHYROLL_ANIME_SIMULCASTS_URL, page++);
			int saved = grabAndUpdateAnimeSimulcasts(url, count);
			if(saved == 0){
				break;
			}else{
				count += saved;
			}
		}
		
		return count;
	}
	
	/**
	 * update anime updated to recent_show
	 * @return
	 */
	public static int saveRecentAnimes(){
		int count = 0;
		int page = 0;		
		boolean run = true;
		while(run){
			String url = StringUtils.format(CRUNCHYROLL_ANIME_UPDATE_URL, page++);
			int saved = grabAndUpdateRecentAnimes(url, count);
			if(saved == 0){
				break;
			}else{
				count += saved;
			}
		}
		
		return count;
	}
	
	/**
	 * grab all dramas
	 * @return
	 */
	public static int saveAllDramas(){
		return saveAnimesOrDramas(TVShow.CATEGORY_DRAMA);
	}
	
	/**
	 * 将部分category='Drama'更新为ChineseTV等
	 * @return
	 */
	public static int updateDramaCountryCategory(){
		int count = 0;
		for (Map.Entry<String, String> entry : dramaCountryCates.entrySet()) {
			String genre = entry.getKey();
			String category = entry.getValue();
			int page = 0;
			while(true){
				String url = StringUtils.format(CRUNCHYROLL_DRAMA_GENRE_URL, page++, genre);
				int updated = grabAndUpdateCategory(url, category);
				if(updated == 0){
					break;
				}else{
					count += updated;
				}
			}			
		}
		
		return count;
	}
		
	private static int grabAndUpdateCategory(String url, String category) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("li.group-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<String> ids = new ArrayList<String>();
		for (Element ele : lis) {
			String vid = ele.attr("group_id");
			if( ! vid.isEmpty()){
				String showId = SiteConstants.ID_PREFIX_CRUNCHYROLL + vid;	
				ids.add(showId);
			}		
		}
		
		//update to db
		int count = ids.size();		
		if(count > 0){
			SqlSession session = DefaultDBHelper.getBatchSqlSession();
			try {
				TVShowMapper mapper = session.getMapper(TVShowMapper.class);	
				
				for (String showId : ids) {
					mapper.updateCategory(showId, category);
				}
				
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}

	private static int saveAnimesOrDramas(String category){
		String urlFormat = null;
		if(TVShow.CATEGORY_ANIME.equals(category)){
			urlFormat = CRUNCHYROLL_ANIME_URL_FORMAT;
		}else if(TVShow.CATEGORY_DRAMA.equals(category)){
			urlFormat = CRUNCHYROLL_DRAMA_URL_FORMAT;
		}else{
			return 0;
		}
		
		int count = 0;
		int page = 0;		
		boolean run = true;
		while(run){
			String url = StringUtils.format(urlFormat,  page++);		
			int saved = grabAndSaveTVShows(url, category, count);			
			if(saved == 0){
				break;
			}else{
				count += saved;
			}
		}
				
		return count;
	}	
	
	private static int grabAndUpdateRecentAnimes(String url, int index) {	
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("li.group-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<String> ids = new ArrayList<String>();
		for (Element ele : lis) {
			String vid = ele.attr("group_id");
			if( ! vid.isEmpty()){
				String showId = SiteConstants.ID_PREFIX_CRUNCHYROLL + vid;
				ids.add(showId);
			}
		}
		
		int count = ids.size();
		if(count > 0){					
			Date version = DateUtils.getNow();
			
			SqlSession session = DefaultDBHelper.getSqlSession(false);
			try {
				TVShowMapper mapper = session.getMapper(TVShowMapper.class);
				RecentShowMapper recentMapper = session.getMapper(RecentShowMapper.class);
				
				for (String showId : ids) {
					RecentShow rs = new RecentShow();
					rs.setSortIndex(++index);
					rs.setVersion(version);
					rs.setShowId(showId);
					
					TVShow tmpTV = mapper.selectByPrimaryKey(showId);
					if(tmpTV != null){
						QueryExample ex = new QueryExample();
						ex.or().addCriterion("show_id=", showId);
						List<RecentShow> recents = recentMapper.selectByExample(ex);
						if(recents.size() > 0){
							rs.setId(recents.get(0).getId());
							recentMapper.updateByPrimaryKey(rs);
						}else{
							rs.setId(UUID.randomUUID().toString());
							recentMapper.insert(rs);
						}
					}
				}
				
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}

	private static int grabAndUpdateAnimeSimulcasts(String url, int index) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("li.group-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<String> showIds = new ArrayList<String>();
		for (Element ele : lis) {
			String vid = ele.attr("group_id");
			if( ! vid.isEmpty()){
				String showId = SiteConstants.ID_PREFIX_CRUNCHYROLL + vid;
				showIds.add(showId);
			}			
		}
		
		int count = showIds.size();
		if(count > 0){
			Date version = DateUtils.getNow();
			SqlSession session = DefaultDBHelper.getSqlSession(false);
			
			try {
				TVShowMapper mapper = session.getMapper(TVShowMapper.class);
				
				for (String showId : showIds) {
					TVShowToday tt = new TVShowToday();
					tt.setVersion(version);
					tt.setSortIndex(++index);
					tt.setShowId(showId);					
					
					TVShow tmpTV = mapper.selectByPrimaryKey(showId);				
					if(tmpTV != null){				
						TVShowTodayMapper todayMapper = session.getMapper(TVShowTodayMapper.class);
						QueryExample ex = new QueryExample();
						ex.or().addCriterion("show_id=", showId);
						List<TVShowToday> todays = todayMapper.selectByExample(ex);
						if(todays.size() > 0){
							tt.setId(todays.get(0).getId());
							todayMapper.updateByPrimaryKey(tt);
						}else{
							tt.setId(UUID.randomUUID().toString());
							todayMapper.insert(tt);
						}						
					}
				}
				
				
				session.commit();
			} finally {
				session.close();
			}
		}
		
		return count;
	}

	private static int grabAndUpdateTVShowGenres(String url, String genre) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("li.group-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<String> showIds = new ArrayList<String>();
		for (Element ele : lis) {
			String vid = ele.attr("group_id");
			if( ! vid.isEmpty()){
				String showId = SiteConstants.ID_PREFIX_CRUNCHYROLL + vid;			
				showIds.add(showId);
			}
		}	
		
		int aff = 0;
		if(showIds.size() > 0){		
			SqlSession session = DefaultDBHelper.getSqlSession(false);
			
			try {
				TVShowMapper mapper = session.getMapper(TVShowMapper.class);
				
				for (String showId : showIds) {
					TVShow tmpTV = mapper.selectByPrimaryKey(showId);
					if(tmpTV != null){				
						aff++;
						
						TVShowGenreMapper genreMapper = session.getMapper(TVShowGenreMapper.class);
						// genre 是否已经关联？
						QueryExample genreEx = new QueryExample();
						genreEx.or().addCriterion("show_id = ", showId)
							.addCriterion("genre_name = ", genre);
						int genreRefCount = genreMapper.countByExample(genreEx);
						boolean isGenreRefed = genreRefCount > 0;
						
						if( ! isGenreRefed){
							String genreName = tmpTV.getGenre();
							String updateGenre = null;
							if(StringUtils.isNullOrEmpty(genreName)){
								updateGenre = genre;
							}else{
								updateGenre = genreName + "," + genre;
							}												
							
							mapper.updateGenre(showId, updateGenre);
							
							TVShowGenre tvShowGenre = new TVShowGenre();
							tvShowGenre.setId(UUID.randomUUID().toString());
							tvShowGenre.setShowId(showId);
							tvShowGenre.setGenreName(genre);
							genreMapper.insert(tvShowGenre);
						}			
					}
				}			
				
				session.commit();
			}finally{
				session.close();
			}
		}
		
		return aff;
	}

	private static int grabAndSaveTVShows(String url, String category, int index) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("li.group-item");
		if(lis.isEmpty()){
			return 0;
		}		
		
		int count = 0;		
		for (Element ele : lis) {
			if(grabAndSaveTVShowFromElement(ele, category, count + index)){
				count++;
			}
		}
		
		return count;
	}
	
	private static boolean grabAndSaveTVShowFromElement(Element element, String category, int sortIndex){
		String vid = element.attr("group_id");
		Element as = element.select("div > a").first();
		if(as != null){
			String img = as.select("span.img-holder > img").attr("src");
			String title = as.select("span.series-title").text().trim();			
			String url = DOMAIN + as.attr("href");					
			
			TVShow show = new TVShow();
			show.setId(SiteConstants.ID_PREFIX_CRUNCHYROLL + vid);
			show.setVideoId(vid);
			//anime/drama category
			show.setCategory(category);
			show.setName(title);
			show.setTitleInitial(StringUtils.getInitial(title));
			show.setThumbnail(img);			
			show.setItemUrl(url);
			show.setWebsite(SiteConstants.WEBSITE_CRUNCHYROLL);
			show.setArea(StandarAreas.DEFAULT);
			
			List<TVEpisode> episodes = obtainTVEpisodes(show);
			return saveTV2DB(show, episodes, sortIndex);							
		}		
		
		return false;
	}

	/**
	 * 抓取tv show相关的episodes, 并完善tv show本身
	 * @param show
	 * @return
	 */
	private static List<TVEpisode> obtainTVEpisodes(TVShow show) {		
		List<TVEpisode> epList = new ArrayList<TVEpisode>();
		
		Document doc = getDocument(show.getItemUrl());
		if(doc == null){
			return epList;
		}
		
		//大图
		String img = doc.select("#sidebar_elements > li > img").eq(0).attr("src");
		if( ! img.isEmpty()) {
			show.setBannerImgUrl(img);
		}		
		
		//score
		String scorecontent = doc.select("#showview_about_rate_widget").attr("content");	
		show.setScore(BigDecimal.valueOf(Double.parseDouble(scorecontent) * 2));
		//描述
		String desc = doc.select("#sidebar_elements > li > p.description").text().trim();
		if(desc.endsWith("... more")){
			desc = desc.substring(0, desc.length() - 8);
		}
		if(desc.length() >= 2000){
			desc = desc.substring(0, 2000);
		}
		show.setDescription(desc);		
		
		//season count
		Elements seasons = doc.select("#showview_content_videos > ul.list-of-seasons > li.season");		
		int seasonCount = seasons.size();
		show.setSeasonCount(seasonCount);		
		String showId = show.getId();
		
		int round = 0;
		for (Element eles : seasons) {
			//抽取第几季信息
			int sc = 0;
			String st = eles.select("a.season-dropdown").text().trim();
			if( ! st.isEmpty()){
				String[] ss = st.split("\\s+");
				sc = NumberUtils.parseInt(ss[ss.length - 1]);
			}
			
			if(sc == 0){
				sc = seasonCount - round;
			}
			
			//抓取每一季
			Elements slis = eles.select("ul.portrait-grid > li.group-item > div.wrapper > a.episode");
			for (Element eleli : slis) {
				String image = eleli.select("img.landscape").attr("src");
				if(image.equals(COMING_SOON_IMAGE)){
					continue;
				}			
				
				String epUrl = DOMAIN + eleli.attr("href");
					
				TVEpisode ep = obtainEpisodeDetail(epUrl, sc);
				
				if(ep != null){
					ep.setUrl(epUrl);
					ep.setId(showId + "E" + ep.getSeason() + "E" + ep.getPart());	
					ep.setShowId(showId);
					ep.setWebsite(SiteConstants.WEBSITE_CRUNCHYROLL);				
					
					if(image.isEmpty()){
						image = eleli.select("img.landscape").attr("data-thumbnailurl");
					}					
					ep.setThumbnail(image);
					
					epList.add(ep);
				}
			}
			
			round++;
		}
		
		return epList;
	}


	private static TVEpisode obtainEpisodeDetail(String epUrl, int season) {				
		Document doc = getDocument(epUrl);
		if(doc == null){
			return null;
		}
		
		Elements h4s = doc.select("#showmedia_about_media > h4");
		if(h4s.size() < 2){
			return null;
		}		
		
		TVEpisode ep = new TVEpisode();
		String desc = doc.select("#showmedia_about_info > p.description").text().trim();		
		if(desc.endsWith("... more")){
			desc = desc.substring(0, desc.length() - 8);
		}
		
		if(desc.length() >= 2000){
			desc = desc.substring(0, 2001);
		}
		ep.setIntroduction(desc);		
		
		String ss = h4s.get(1).text().trim();
		ss = ss.replace("Season", "").replace("Episode", "");			
		String[] ass = ss.split("\\s*,\\s*");	
		
		if(ass.length == 1){
			ep.setSeason(season);
			ep.setPart(NumberUtils.parseInt((ass[0]).trim(), 1));
		}else{
			int sc = NumberUtils.parseInt((ass[0]).trim());
			if(sc == 0){
				sc = season;
			}
			ep.setSeason(sc);
			ep.setPart(NumberUtils.parseInt((ass[1]).trim(), 1));
		}		
		
		//name
		String name = doc.select("#showmedia_about_name").text().trim();
		if(name.isEmpty()){
			ep.setEpisodeName("Episode " + ep.getPart());
		}else{
			ep.setEpisodeName(name.replace("“", "").replace("”", ""));
		}
		
		//publish date
		Elements divs = doc.select("#showmedia_about_info_details > div");
		for (Element ele : divs) {
			String t = ele.ownText().trim();
			if("Availability for free users:".equals(t)){				
				//Aug 5, 2012
				String d = ele.select("span").text().trim();
				if("Now".equals(d)){
					ep.setAirDate(DateUtils.getNow());
				}else if("Not available".equals(d)){
					ep.setAirDate(DateUtils.getNow());
				}else{
					String[] dds = d.split("(?:,\\s*|\\s+)");
					if(dds.length != 3){
						ep.setAirDate(DateUtils.getNow());
					}else{
						int month = DateUtils.getShortMonth(dds[0]);
						String date = month + "/" + dds[1] + "/" + dds[2];
						SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy");
						ep.setAirDate(DateUtils.parse(format, date));
					}
				}
				
				break;
			}
		}
		
		return ep;
	}	

	private static boolean saveTV2DB(TVShow show, List<TVEpisode> episodes,
			int index) {	
		int size = episodes.size();	
		if(size == 0){
			return false;
		}
		
		Date now = DateUtils.getNow();
		String showId = show.getId();
		show.setSortIndex(index + 1);
		show.setAirBeginDate(episodes.get(0).getAirDate());
		if(size > 1){
			show.setAirEndDate(episodes.get(size - 1).getAirDate());
		}
		show.setVersion(now);
		show.setEpisodeCnt(episodes.size());
		
		SqlSession session = DefaultDBHelper.getSqlSession();
		
		try {
			TVShowMapper mapper = session.getMapper(TVShowMapper.class);
			TVShow tmpTV = mapper.selectByPrimaryKey(showId);
			if(tmpTV != null){
				//System.err.println("crash: " + show.getId() + " | url: " + show.getItemUrl());
				mapper.updateByPrimaryKey(show);
			}else{
				mapper.insert(show);
			}
			
			//episodes			
			TVEpisodeMapper episodeMapper = session.getMapper(TVEpisodeMapper.class);
			for (TVEpisode ep : episodes) {
				if(episodeMapper.selectByPrimaryKey(ep.getId()) != null){					
					episodeMapper.updateByPrimaryKey(ep);
				}else{
					episodeMapper.insert(ep);
				}
			}
			
			// 统计seasons
			QueryExample seasonEx = new QueryExample();
			seasonEx.or().addCriterion("tv_season.show_id = ", showId);
			List<Integer> seasons = episodeMapper.selectSeasons(showId);
			TVSeasonMapper seasonMapper = session.getMapper(TVSeasonMapper.class);
			seasonMapper.deleteByExample(seasonEx);
			for (Integer seasonNo : seasons) {
				seasonEx.clear();
				seasonEx.or().addCriterion("tv_season.show_id = ", showId)
					.addCriterion("tv_season.season_no = ", seasonNo);
				if (seasonMapper.countByExample(seasonEx) < 1) { // 不存在
					TVSeason season = new TVSeason();
					season.setId(UUID.randomUUID().toString());
					season.setShowId(showId);
					season.setSeasonNo(seasonNo);
					season.setVersion(Calendar.getInstance().getTime());
					seasonMapper.insert(season);
				}
			}
			
			session.commit();
			return true;
		} finally {
			session.close();
		}		
	}
	
	private static Map<String, String> cookies = new HashMap<String, String>();	
	static {
		fillTheCookies();
	}
	
	private static void fillTheCookies(){
		try {
			HttpURLConnection conn = (HttpURLConnection)new URL(DOMAIN).openConnection();
			conn.connect();
			
			for (int i=0; ; i++) {
		        String headerName = conn.getHeaderFieldKey(i);
		        String headerValue = conn.getHeaderField(i);

		        if (headerName == null && headerValue == null) {		            
		            break;
		        }
		        if (headerName == null) {
		            // The header value contains the server's HTTP version
		        }
		        
		        if("Set-Cookie".equals(headerName)){
		        	String cookie = headerValue.substring(0, headerValue.indexOf(";"));
		        	String[] ss = cookie.split("=");
		        	if(ss.length == 2){
		        		cookies.put(ss[0], ss[1]);
		        	}
		        }
		    }
			
		} catch (Exception e) {
			logger.error("error happen when try to get the Crunchyroll Cookies: ", e);
		}	
	}
		
	private static Document getDocument(String url){
		try {
			//return Jsoup.connect(url).timeout(CONNECT_READ_TIMEOUT).userAgent(USER_AGENT).get();
			Connection conn = Jsoup.connect(url).timeout(CONNECT_READ_TIMEOUT).userAgent(USER_AGENT).referrer(REFERRER);
			if( ! cookies.isEmpty()){
				for (Map.Entry<String, String> entry : cookies.entrySet()) {
					conn.cookie(entry.getKey(), entry.getValue());
				}
			}
			
			return conn.get();
			
		} catch (IOException e) {
			logger.error("error happened when fetch " + url + " webpage: " + e);
		}
		
		return null;
	}
	
	

	public static void main(String[] args) {
		saveAllDramas();
	}
}
